import os
import datetime
import copy
import shutil
from collections import OrderedDict

from utils.file_md5 import file_md5

import gevent

from watchdog_gevent import Observer
from watchdog.events import FileSystemEventHandler

from app.utils.ws_data import *

from app_instance import app

import sqlalchemy

class FileRecvHistoryModel(app.db.Model):
    id = sqlalchemy.Column(sqlalchemy.INTEGER, primary_key=True)
    file_name = sqlalchemy.Column(sqlalchemy.String(50), unique=True)
    checksum = sqlalchemy.Column(sqlalchemy.String(50))
    create_t = sqlalchemy.Column(sqlalchemy.String(50))

app.db.create_all()

class WSState(FileSystemEventHandler):
    def __init__(self,
                 client_id,
                 main_loop,
                 data_dir,
                 inbox_dir,
                 outbox_dir,
                 logger,
                 task_timeout_secs = 120,
                 is_server=False,
                 chunk_size=5 * 1000 * 1024):

        self.client_id = client_id
        self.main_loop = main_loop
        self.logger = logger
        self.is_server = is_server

        self.chunk_size = chunk_size
        self.data_dir = os.path.abspath(data_dir)
        self.ignored_dir = os.path.join(self.data_dir, 'ignored')
        self.tmp_dir = os.path.join(self.data_dir, 'tmp')
        self.inbox = os.path.abspath(inbox_dir)
        self.outbox = os.path.abspath(outbox_dir)

        self.datetime_format = '%Y-%m-%d %H:%M:%S.%f'

        self.unack_task_timeout = datetime.timedelta(seconds=task_timeout_secs)

        self.pending_files = set()
        self.finished_files = set()
        self.finished_unfinished_files = set()
        self.finished_tasks = set()

        if not os.path.exists(self.inbox):
            os.makedirs(self.inbox)
        if not os.path.exists(self.outbox):
            os.makedirs(self.outbox)
        if not os.path.exists(self.tmp_dir):
            os.makedirs(self.tmp_dir)
        if not os.path.exists(self.ignored_dir):
            os.makedirs(self.ignored_dir)

        self.tasks = OrderedDict()
        self.unack_tasks = {}

        self.clients_state = {}

        self.task_record_path = os.path.join(self.data_dir, 'tasks.json')
        if os.path.exists(self.task_record_path):
            try:
                with open(self.task_record_path, 'rb') as fd:
                    self.tasks = OrderedDict(json.load(fd))
            except:
                pass

        self.unack_task_record_path = os.path.join(self.data_dir, 'unack_tasks.json')
        if os.path.exists(self.unack_task_record_path):
            try:
                with open(self.unack_task_record_path, 'rb') as fd:
                    self.unack_tasks = json.load(fd)
            except:
                pass

        self.unfinished_states = {}
        self.transient_tasks = []

        for root, dir, files in os.walk(self.tmp_dir):
            for file in files:
                if file.endswith('.unfinished'):
                    try:
                        with open(os.path.join(root, file)) as fd:
                            state = json.load(fd)
                            if 'file_name' in state:
                                state['modify_t'] = self.now()
                                file_name = state['file_name']
                                self.unfinished_states[file_name] = state
                                self.logger.debug('adding unfinished task %s' % file_name)
                                self.add_transient_task(self.make_unfinished_task(state))
                    except Exception as ex:
                        self.logger.error(ex)

        self.logger.debug("scaning fold %s..." % self.outbox)
        with os.scandir(self.outbox) as it:
            for entry in it:
                if entry.is_file() and \
                        not entry.name.startswith('.') and \
                        not entry.name.endswith('.unfinished'):
                    self.add_file_task(entry.path, False)

        self.sort_tasks()
        self.sync_tasks()

        self.timer = main_loop.timer(3, 3)
        self.timer.start(self.tick)

        observer = Observer()
        observer.schedule(self, self.outbox, recursive=False)
        observer.start()

    def active_client(self, client_id):
        initial = False
        if client_id in self.clients_state:
            initial = not self.clients_state[client_id][1]
            self.clients_state[client_id][1] = datetime.datetime.now()
        else:
            initial = True
            self.clients_state[client_id] = [True, datetime.datetime.now()]
        return initial

    def deactive_client(self, client_id):
        if client_id in self.clients_state:
            self.clients_state[client_id][0] = False

    def refresh_unack_task(self, client_id):
        pass

    @staticmethod
    def __is_push_task_exists(tasks, file_name):
        if file_name in tasks:
            task = tasks[file_name]
            if 'opcode' in task:
                if task['opcode'] == WSOPCode.PUSH_FILE.value or task['opcode'] == WSOPCode.PUSH_CHUNKS.value:
                    return True

    def is_unfinished_task(self, file_name):
        return file_name in self.unfinished_states

    def is_push_task_exists(self, file_name):
        for task in self.transient_tasks:
            if task['file_name'] == file_name and \
                    (task['opcode'] == WSOPCode.PUSH_FILE.value or task['opcode'] == WSOPCode.PUSH_CHUNKS.value):
                return True

        if self.__is_push_task_exists(self.tasks, file_name):
            return True

        if self.__is_push_task_exists(self.unack_tasks, file_name):
            task = copy.deepcopy(self.unack_tasks[file_name])
            self.return_task(task)
            return True

        return False

    def is_finished_task(self, file_name):
        try:
            history = FileRecvHistoryModel.query.filter_by(file_name=file_name).first()

            if history != None:
                self.logger.warn('task %s already finished' % file_name)
                return True
        except:
            pass
        return False

    def is_unfinished_file(self, file_name):
        return file_name in self.unfinished_states

    def is_task_timeout(self, task):
        assert 'modify_t' in task
        modify_t = datetime.datetime.strptime(task['modify_t'], self.datetime_format)
        diff = datetime.datetime.now() - modify_t
        return diff > self.unack_task_timeout

    def fetch_unfinished_file_task(self, file_name):
        if file_name in self.unfinished_states:
            return self.unfinished_states[file_name]

        tasks = []
        for task in self.transient_tasks:
            if task['file_name'] == file_name:
                tasks.append(task)

        unfinished_task = None
        if len(tasks) > 0:
            unfinished_task = tasks.pop(0)
        for task in tasks:
            self.transient_tasks.remove(task)

        if unfinished_task is None:
            if file_name in self.unfinished_states:
                unfinished_task = self.make_unfinished_task(self.unfinished_states[file_name])

        return unfinished_task

    def is_finished_file(self, file_name):
        return file_name in self.finished_files

    def tick(self):
        pending_files, self.pending_files = self.pending_files, set()
        for file_path in pending_files:
            self.add_file_task(file_path)

        finished_files, self.finished_files = self.finished_files, set()
        for file_name in finished_files:
            file_path = self.get_outbox_file_path(file_name)
            try:
                os.remove(file_path)
                self.logger.debug('file %s removed' % file_name)
            except FileNotFoundError:
                pass
            except Exception as ex:
                self.finished_files.add(file_name)
                self.logger.error('exception %s happend while remove file %s' % (ex, file_path))

        finished_unfinished_files, self.finished_unfinished_files = self.finished_unfinished_files, set()
        for file_name in finished_unfinished_files:
            self.finish_unfinished_state(file_name)

        for file_name in self.finished_tasks:
            # pass empty client id to finished empty clients id task
            self.finish_task(file_name, '')

        self.finished_tasks.clear()

        now = datetime.datetime.now()
        # currently not work
        for client_state in self.clients_state:
            if client_state[0] == True:
                diff = now - client_state[1]
                if diff > self.unack_task_timeout:
                    client_state[0] = False

        timeout_tasks = []
        for file_name, unack_task in self.unack_tasks.items():
            if 'modify_t' in unack_task:
                modify_t = datetime.datetime.strptime(unack_task['modify_t'], self.datetime_format)
                diff = datetime.datetime.now() - modify_t
                if diff > self.unack_task_timeout:
                    self.logger.warn('unack task %s timeout(%ss)' % (file_name, self.unack_task_timeout))
                    # downgrade the task priority
                    timeout_tasks.append(unack_task)
            else:
                self.logger.error('modify_t not in unack task')

        for timeout_task in timeout_tasks:
            clients_id = timeout_task['clients_id']
            self.logger.debug('returning task %s, clients id %s' % (timeout_task['file_name'], ','.join(clients_id)))
            task = copy.deepcopy(timeout_task)
            self.return_task(task, sync=False)

        for file_name, unfinished_state in self.unfinished_states.items():
            if self.is_finished_task(file_name):
                self.finished_tasks.add(file_name)

            if 'modify_t' in unfinished_state:
                modify_t = datetime.datetime.strptime(unfinished_state['modify_t'], self.datetime_format)
                diff = datetime.datetime.now() - modify_t
                if diff > self.unack_task_timeout:
                    self.logger.debug('unfinished task %s timeout(%ss)' % (unfinished_state['file_name'], self.unack_task_timeout))
                    self.add_transient_task(self.make_unfinished_task(unfinished_state))
                    unfinished_state['modify_t'] = self.now()

        self.sort_tasks()
        self.sync_tasks()
        self.sync_unack_tasks()

    def make_unfinished_task(self, unfinished_state):
        ack_chunks = unfinished_state['ack_chunks']
        size = unfinished_state['size']
        chunks = self.calc_unack_chunks(size, self.chunk_size, ack_chunks)
        self.logger.debug('making unfinished task %s' % unfinished_state['file_name'])
        return {
            'file_name': unfinished_state['file_name'],
            'clients_id': unfinished_state['clients_id'],
            'opcode': WSOPCode.PUSH_CHUNKS.value if self.is_server else WSOPCode.PULL_CHUNKS.value,
            'size': size,
            'checksum': unfinished_state['checksum'],
            'chunks':  chunks,
            'transient': True,
            'create_t': unfinished_state['create_t'],
            'modify_t': self.now()
        }

    def now(self):
        return datetime.datetime.now().strftime(self.datetime_format)

    def from_time_str(self, time_str):
        return datetime.datetime.strptime(time_str, self.datetime_format)

    def to_time_str(self, dt):
        return dt.strftime(self.datetime_format)

    def update_unack_task_modify_t(self, file_name):
        if file_name in self.unack_tasks:
            self.unack_tasks[file_name]['modify_t'] = self.now()

    def is_file_task_exists(self, file_name, client_id):
        if file_name in self.unack_tasks:
            task = self.unack_tasks[file_name]
            if client_id in task['clients_id']:
                return True

        if file_name in self.tasks:
            task = self.tasks[file_name]
            return client_id in task['clients_id']

        return False

    @staticmethod
    def split_chunk(offset, size, chunk_size):
        chunks = []
        o = offset
        while o < size:
            s = min(chunk_size, size - o)
            chunks.append((o, s))
            o += s
        return chunks

    @staticmethod
    def calc_unack_chunks(size, chunk_size, ack_chunks):
        unack_chunks = []
        ack_offset = 0
        for ack_chunk in ack_chunks:
            if ack_chunk[0] > ack_offset:
                unack_chunks.append(ack_offset, ack_chunk[0])
            ack_offset = ack_chunk[0] + ack_chunk[1]
        if size > ack_offset:
            unack_chunks.append((ack_offset, size - ack_offset))
        return unack_chunks

    @staticmethod
    def is_chunks_finished(size, ack_chunks):
        if size == 0:
            return True
        ack_offset = 0
        for ack_chunk in ack_chunks:
            if ack_chunk[0] > ack_offset:
                return False
            ack_offset = ack_chunk[0] + ack_chunk[1]
        return ack_offset >= size

    def parse_clients(self, file_path):
        file_name = os.path.basename(file_path)
        if file_name.startswith('oa_'):
            parts = file_name.split('_')
            if len(parts) > 2:
                clients_part = parts[2]
                return [item for item in clients_part.split(';') if len(item) > 0]
        elif file_name.endswith('.xml'):
            if not file_name.endswith('_other.xml'):
                with open(file_path, 'rb') as fd:
                    h = fd.read(150).decode('utf-8')
                    import re
                    m = re.search('data_jsdw="([;0-9]+)"', h)
                    if m:
                        clients_part = m.group(1)
                        return [item for item in clients_part.split(';') if len(item) > 0]
        try:
            ignored_path = os.path.join(self.ignored_dir, os.path.basename(file_path))
            shutil.move(file_path, ignored_path)
        except Exception as ex:
            self.logger.error(ex)
        return []

    def update_task(self, task_info):
        task = None
        file_name = task_info['file_name']
        if file_name in self.tasks:
            task = self.tasks[file_name]
        elif file_name in self.unack_tasks:
            task = self.unack_tasks[file_name]
        else:
            self.add_task(task_info)
            
        if task:
            task['size'] = task_info['size']
            task['checksum'] = task_info['checksum']

    def make_task(self, file_path):
        if os.path.exists(file_path):
            try:
                file_name = os.path.basename(file_path)
                md5 = file_md5(file_path)
                clients = self.parse_clients(file_path)

                if len(clients) <= 0:
                    self.logger.warn('ignore invalid file %s' % file_path)
                    return None

                if self.is_server:
                    clients = list(set(clients) - set([self.client_id]))
                    # ignore task
                    if len(clients) <= 0:
                        self.logger.warning('empty receivers in server side, %s' % file_name)
                        return None
                else:
                    clients = [self.client_id]

                return \
                    {
                        'file_name': file_name,
                        'clients_id': clients,
                        'opcode': WSOPCode.PULL_FILE.value if self.is_server else WSOPCode.PUSH_FILE.value,
                        'size': os.path.getsize(file_path),
                        'checksum': md5,
                        'create_t': self.now(),
                        'modify_t': self.now()
                    }
            except Exception as ex:
                self.logger.error(ex)
        return None

    def update_file_task(self, file_path):
        task = self.make_task(file_path)
        if task:
            self.main_loop.run_callback(self.update_task, task)

    def add_file_task_async(self, file_path, sync=True):
        self.main_loop.run_callback(self.add_file_task, file_path, sync)

    def add_file_task(self, file_path, sync=True):
        try:
            if os.path.exists(file_path):
                task = self.make_task(file_path)
                if task:
                    self.add_task(task, sync=sync)
        except Exception as ex:
            self.logger.error(ex)
            self.pending_files.add(file_path)

    # override FileSystemEventHnadler
    def on_created(self, event):
        if not event.is_directory:
            file_path = event.src_path
            if not os.path.basename(file_path).startswith('.'):
                self.logger.debug('file %s created' % file_path)
                self.add_file_task_async(file_path, True)

    def on_modified(self, event):
        if not event.is_directory:
            file_path = event.src_path
            if not os.path.basename(file_path).startswith('.'):
                self.logger.debug('file %s modified' % file_path)
                self.main_loop.run_callback(self.update_file_task, file_path)

    def get_tmp_file_path(self, file_name):
        return os.path.join(self.tmp_dir, file_name)

    def get_dst_file_paths(self, file_name):
        dst_paths = []
        tmp_file_path = self.get_tmp_file_path(file_name)
        if os.path.exists(tmp_file_path):
            if self.is_server:
                clients = self.parse_clients(tmp_file_path)
                if self.client_id in clients:
                    dst_paths.append(os.path.join(self.inbox, file_name))
                if len(set(clients) - set([self.client_id])) > 0:
                    dst_paths.append(os.path.join(self.outbox, file_name))
            else:
                dst_paths.append(os.path.join(self.inbox, file_name))
        else:
            self.logger.error('temporary file %s not exists' % tmp_file_path)
        return dst_paths

    def get_outbox_file_path(self, file_name):
        return os.path.join(self.outbox, file_name)

    def add_unfinished_state(self, state):
        exists = False
        state['create_t'] = self.now()
        if 'file_name' in state:
            file_name = state['file_name']
            if file_name in self.unfinished_states:
                state_ = self.unfinished_states[file_name]
                state_['size'] = state['size']
                state_['checksum'] = state['checksum']
                state_['modify_t'] = self.now()
                state = state_
                exists = True
            else:
                state['ack_chunks'] = []
                state['modify_t'] = self.now()
                self.unfinished_states[file_name] = state
                exists = False

            self.sync_unfinished_state(state)

        return not exists

    def update_unfinished_state(self, file_name, chunk_item):
        if file_name in self.unfinished_states:
            state = self.unfinished_states[file_name]
            state['modify_t'] = self.now()
            if 'ack_chunks' in state:
                ack_chunks = state['ack_chunks']
                handled = False
                if len(ack_chunks) > 0:
                    last_chunk = ack_chunks[-1]
                    if last_chunk[0] + last_chunk[1] == chunk_item[0]:
                        ack_chunks[-1] = (last_chunk[0], chunk_item[0] + chunk_item[1])
                        handled = True
                if not handled:
                    ack_chunks.append(chunk_item)
                ack_chunks.sort(key=lambda x: x[0])

                if self.is_chunks_finished(state['size'], state['ack_chunks']):
                    return True

            self.sync_unfinished_state(state)
        return False

    def finish_unfinished_state(self, file_name):
        if file_name in self.unfinished_states:
            state = self.unfinished_states[file_name]
            try:
                tmp_file_path = self.get_tmp_file_path(file_name)
                if os.path.exists(tmp_file_path):
                    dst_file_paths = self.get_dst_file_paths(file_name)
                    for dst_file_path in dst_file_paths:
                        shutil.copy(tmp_file_path, dst_file_path)
                    os.remove(tmp_file_path)

                tmp_unfinished_file_path = self.get_unfinished_file_path(state['file_name'])
                if os.path.exists(tmp_unfinished_file_path):
                    os.remove(tmp_unfinished_file_path)
                self.unfinished_states.__delitem__(file_name)
            except Exception as ex:
                self.finished_unfinished_files.add(file_name)
                self.logger.error(ex)

    def add_finish_task_record(self, state):
        try:
            app.db.session.add(
                FileRecvHistoryModel(file_name=state['file_name'], checksum=state['checksum'], create_t=self.now()))
            app.db.session.commit()
        except Exception as ex:
            self.logger.error(ex)

    def get_unfinished_file_path(self, file_name):
        return os.path.join(self.tmp_dir, file_name + '.unfinished')

    def dump(self, file_path, json_data):
        '''
        fobject = FileObjectThread(open(file_path, 'wb'))
        fobject.write(json_data.encode())
        fobject.close()
        '''
        try:
            fd = open(file_path, 'wb')
            fd.write(json_data.encode())
            fd.close()
        except Exception as ex:
            self.logger.error(ex)

    def sync_unfinished_state(self, state):
        if 'file_name' in state:
            file_name = state['file_name']
            json_data = json.dumps(state)
            gevent.spawn(self.dump, self.get_unfinished_file_path(file_name), json_data)

    def sync_tasks(self):
        json_data = json.dumps(self.tasks)
        gevent.spawn(self.dump, self.task_record_path, json_data)

    def sync_unack_tasks(self):
        json_data = json.dumps(self.unack_tasks)
        gevent.spawn(self.dump, self.unack_task_record_path, json_data)

    def get_unfinished_state(self, file_name):
        if file_name in self.unfinished_states:
            return self.unfinished_states[file_name]
        return None

    def get_unfinished_states(self):
        return self.unfinished_states

    def add_task(self, task, sync=True):
        assert 'file_name' in task
        file_name = task['file_name']

        if file_name.endswith('_netTest.xml'):
            task['create_t'] = datetime.datetime(1970, 1, 1, 0, 0, 0).strftime(self.datetime_format)

        unhandle_clients_id = task['clients_id']

        if len(unhandle_clients_id) == 0:
            self.logger.warning('task %s clients id is empty' % task['file_name'])
            return

        if file_name in self.unack_tasks:
            task_ = self.unack_tasks[file_name]
            unhandle_clients_id = set(unhandle_clients_id) - set(task_['clients_id'])

        if file_name in self.tasks:
            task_ = self.tasks[file_name]
            unhandle_clients_id = set(unhandle_clients_id) - set(task_['clients_id'])

        if len(unhandle_clients_id) > 0:
            if file_name in self.tasks:
                task_ = self.tasks[file_name]
                task_['clients_id'] = list(set(task_['clients_id']) | set(unhandle_clients_id))
            else:
                self.logger.debug('added new task %s' % file_name)
                self.tasks[file_name] = task
        else:
            return

        if sync:
            self.sort_tasks()
            self.sync_tasks()

    def sort_tasks(self):
        self.tasks = OrderedDict(sorted(self.tasks.items(), key=lambda t: t[1]['create_t']))

    def add_transient_task(self, task):
        assert 'file_name' in task
        assert 'opcode' in task
        if 'create_t' not in  task:
            task['create_t'] = self.now()
        task['transient'] = True
        task['modify_t'] = self.now()
        self.transient_tasks.append(task)

        self.transient_tasks.sort(key=lambda t: t['create_t'])

    def task_count(self):
        return len(self.tasks)

    def transient_task_count(self):
        return len(self.transient_tasks)

    def fetch_task(self, client_id, sync=True, prefer_local=False):
        task = None

        if not prefer_local and len(self.transient_tasks) > 0:
            if self.is_server:
                for transient_task in self.transient_tasks:
                    clients_id = transient_task['clients_id']
                    if client_id in clients_id:
                        clients_id.remove(client_id)
                        if len(clients_id) == 0:
                            self.transient_tasks.remove(transient_task)
                        task = copy.deepcopy(transient_task)
                        task['clients_id'] = [client_id]
                        break
            else:
                task = self.transient_tasks[0]
                self.transient_tasks.remove(task)

        if not task and client_id:
            for file_name, task_ in self.tasks.items():
                #if len(task_['clients_id']) == 0:
                #   if file_name not in self.unack_tasks or len(self.unack_tasks[file_name]['clients_id']) == 0:
                #        self.finished_tasks.add(file_name)

                if 'clients_id' in task_ and \
                    client_id in task_['clients_id']:
                    task_['clients_id'].remove(client_id)

                    if len(task_['clients_id']) <= 0:
                        #self.logger.debug('removing task %s' % file_name)
                        self.tasks.__delitem__(file_name)

                    clients_id = []
                    unack_task = None
                    if file_name in self.unack_tasks:
                        unack_task = self.unack_tasks[file_name]
                        clients_id = unack_task['clients_id']
                        clients_id.append(client_id)
                    else:
                        unack_task = copy.deepcopy(task_)
                        self.unack_tasks[file_name] = unack_task
                        clients_id = [client_id]

                    unack_task['modify_t'] = self.now()
                    unack_task['clients_id'] = list(set(clients_id))

                    if file_name not in self.unack_tasks:
                        task_['modify_t'] = self.now()

                    task = copy.deepcopy(task_)
                    task['clients_id'] = [client_id]
                    break

        if task and 'opcode' in task:
            opcode = task['opcode']
            if 'file_name' in task:
                file_name = task['file_name']
                if (self.is_server and opcode == WSOPCode.PULL_FILE.value) or \
                        (not self.is_server and opcode == WSOPCode.PUSH_FILE.value):
                    if not os.path.exists(self.get_outbox_file_path(file_name)):
                        if file_name in self.unack_tasks:
                            self.unack_tasks.__delitem__(task['file_name'])
                        self.logger.info('drop task, file %s not exists' % file_name)
                        task = None

        if task:
            task['modify_t'] = datetime.datetime.now().strftime(self.datetime_format)

        if sync:
            self.sync_tasks()
            self.sync_unack_tasks()

        return task

    def return_task(self, task, sync=True):
        if len(task['clients_id']) == 0:
            self.logger.warning('returning a empty clients id task, file_name: %s' % task['file_name'])
            return

        if 'transient' in task and \
            task['transient'] == True:
            self.add_transient_task(task)
        else:
            file_name = task['file_name']
            if file_name in self.tasks:
                self.logger.debug('add to exist task %s' % file_name)
                exist_task = self.tasks[file_name]
                exist_task['clients_id'] = list(set(task['clients_id']) | set(exist_task['clients_id']))
                self.logger.debug('exist task clients id %s' % ','.join(exist_task['clients_id']))
            else:
                #self.logger.debug('return task %s' % file_name)
                self.tasks[file_name] = task

            if file_name in self.unack_tasks:
                unack_task = self.unack_tasks[file_name]
                for client_id in task['clients_id']:
                    unack_task['clients_id'].remove(client_id)
                if len(unack_task['clients_id']) <= 0:
                    self.unack_tasks.__delitem__(file_name)
            if sync:
                self.sort_tasks()
                self.sync_tasks()
                self.sync_unack_tasks()

    def remove_push_task(self, file_name):
        if file_name in self.tasks:
            self.logger.debug('removing task %s' % file_name)
            self.tasks.__delitem__(file_name)

    def finish_task(self, file_name, client_id):
        if not self.is_server:
            finished_transient_task = []
            for task in self.transient_tasks:
                if task['file_name'] == file_name:
                    finished_transient_task.append(task)
            for task in finished_transient_task:
                self.transient_tasks.remove(task)

        need_sync_unack_tasks = False
        need_sync_tasks = False

        if file_name in self.unack_tasks:
            task = self.unack_tasks[file_name]

            assert 'file_name' in task

            clients_id = task['clients_id']
            if client_id in clients_id:
                self.logger.debug('removing client id %s from %s' % (client_id, file_name))
                clients_id.remove(client_id)

            if len(clients_id) <= 0:
                self.unack_tasks.__delitem__(file_name)
                self.logger.debug('removing unack task %s' % file_name)

            need_sync_unack_tasks = True

        if file_name in self.tasks:
            task = self.tasks[file_name]

            assert 'file_name' in task

            clients = task['clients_id']
            if client_id in clients:
                clients.remove(client_id)

            if len(clients) <= 0:
                self.tasks.__delitem__(file_name)

            need_sync_tasks = True

        if file_name not in self.tasks and file_name not in self.unack_tasks:
            file_path = self.get_outbox_file_path(file_name)
            if os.path.exists(file_path):
                try:
                    os.remove(file_path)
                    self.logger.info('file %s removed' % file_name)
                except Exception as ex:
                    self.finished_files.add(file_name)
                    self.logger.error('exception %s happend while remove file %s' % (ex, file_path))

        if need_sync_unack_tasks:
            self.sync_unack_tasks()
        if need_sync_tasks:
            self.sync_tasks()
